
import os,sys,File,string,time,math,shutil,traceback
from Configuration import Configuration
"""This is used to decide when analysis a file how many lines should skip"""
#skip = {"1a1":1 , "1a2":1, "1a3":1, "1c":1, "1a4":1, "1a6-1024":1, "1a6-2048":1, "1a6-256":1, "1a6-128":1, "1a6-64":1, "1a5":1, "1a6":1,
#        "2a6":1, "3a6":1, "2a6-2048":1, "2a7":1, "3a7":1, "3a7-1024":1, "2a7-1024":1, "3a7-2048":1, "2a7-2048":1, "1a7":1, "1a10":5}
from Util import fm
Configuration = Configuration()
def updateSkipList(year):
    return True
#    for key in skip.keys():
#        skip[key] = 1 + year

def getSkip(expName):
    return Configuration.getMaxYear(expName) + 1
#    if skip.get(expName) != None:
#        return skip[expName]
#    else:
#        return 2
    
def preCompileTime(expName): 
    l = [[()]]
#    empty = True;
#    dir = fm(os.path.join(Configuration.getPath(), expName))
#    print dir;
#    for pth in xrange(1, 11/1):
#        t = [()]
#        filename = "test" + expName + "_result_%d"%pth + ".txt"
#        if os.path.exists(filename) == False:
#            continue
#        file = open(os.path.join(dir,filename))
#        print file;
#        line = file.readline()
#        if(len(line) == 0):
#            return l
#        for i in xrange(1, string.atoi(line) + 1):
#            file.readline()
#            startTime = string.atof(file.readline())
#            for j in xrange(1, getSkip(expName) + 1/1):
#                file.readline()
#            endTime = string.atof(file.readline())
#            end = file.readline() #skip the time line
#            file.readline() #skip the blank row
#            print startTime, endTime
#            t.insert(i - 1,(startTime, endTime))
#        for tuple in t:
#            print tuple
#        l.insert(pth - 1, t)
#        empty = False;
#    if(empty):
#        sys.stderr.write("Time array is empty")
    return l

def printL(l):
    for t in l:
        for tuple in t:
            print tuple

def nextStartTime(endTime, l):
    for t in l:
        for tuple in t:
            if len(tuple) >0 and string.atof(endTime) < string.atof(tuple[0]):
                return string.atof(tuple[0])
    return 0.0

def staticsCPUandMem(expId, expName, lowTurn, upperTurn, lowFactor, upperFactor, year):
    print "staticsCPUandMem", expId, expName, lowFactor
    AnalysisCPUandMem(expId, expName, lowTurn, lowFactor)
    print "AnalysisCPUandMem"
#    import time
#    time.sleep(1)
    simplifyAndInsert(expId, expName, lowTurn, lowTurn, lowFactor, lowFactor)
    print "end of staticsCPUandMem"


def simplifyAndInsert(expId, expName, lowTurn, upperTurn, lowFactor, upperFactor):
#    print "simplify"
    hostname = getHostname()
    dir = Configuration.getPathExp(expName)
    childDir = fm(os.path.join(dir, "analysis"))
    for turn in xrange(lowTurn, upperTurn+1/1):
        for factor in xrange(lowFactor, upperFactor+1/1):
            filename = "%d_%d_"%(expId,factor)+ hostname + ".txt" 
#            print "simplifyAndInsert " + os.path.join(childDir,filename)
            if os.path.exists(fm(os.path.join(childDir,filename))) == False:
                print "file Not Exist", filename
                return
            file = open(fm(os.path.join(childDir,filename)))
            result = open(fm(os.path.join(childDir,filename + "_simplified")), "w")
            lines = file.readlines()
            times = string.split(lines[0])
    #        print times
    #        print math.floor(string.atof(times[0])), math.floor(string.atof(times[1]))
            firsttime = pretime = string.atof(times[0])
            count = 1 
            total1 = total2 = 0
            cputime = "0"
            for i in xrange(1, len(lines)/1):
                tuple = string.split(lines[i])
                curtime = string.atof(tuple[0])
                if len(tuple) == 4:
                    print >> result, curtime - firsttime, tuple[1], tuple[2], tuple[3]
                else :
                    print >> result, curtime - firsttime, tuple[1], tuple[2], 0
                
                if len(tuple) == 4:
                    cputime = tuple[3]
                else: cputime = "0"
            result.close()
            newfilename = fm(Configuration.getFinalCpuMemFileName(expId, expName, lowFactor))
            shutil.copy(fm(os.path.join(childDir,filename + "_simplified")), newfilename)
            os.remove(fm(os.path.join(childDir,filename + "_simplified")));
#            os.system("rm " + os.path.join(childDir,filename + "_simplified"));
            

def getHostname():
    return Configuration.getHostName()

def getPath(expId,expName,factor):
    hostname = getHostname()
    path = Configuration.getPath()
    cpumempath = os.path.join(path, "%d_"%expId+expName+"_%d_"%factor+"cpumem" + hostname)
    dir = os.path.join(path, expName)
    childDir = os.path.join(dir, "analysis")
    return (fm(cpumempath), fm(dir), fm(childDir))

def AnalysisCPUandMem(expId, expName, turn, factor):
    timeArray = preCompileTime(expName)
    hostname = getHostname()
    tuple = getPath(expId,expName,factor)
    cpumempath = tuple[0]
    dir = tuple[1]
    childDir = tuple[2]
    File.newDir(childDir)
    if os.path.exists(cpumempath) == False:
        print cpumempath, "no found"
        return
    cpumem = open(cpumempath)
    for pth in xrange(1, turn + 1/1):
        filename = Configuration.getExpLog(expId, expName, factor)
#        print "AnalysisCPUandMem", filename
        if os.path.exists(filename) == False:
            sys.stderr.write(filename + " no found")
            return
        file = open(filename)
        line = file.readline()
        if len(line) == 0:
            return 
        for i in xrange(factor, factor + 1):
            filename = "%d"%expId + "_%d_"%i + hostname + ".txt"
#            print "t1 resultfile", filename
            result = open(fm(os.path.join(childDir,filename)), "w")
            print fm(os.path.join(childDir,filename));
#            print "start ",
            file.readline()
            startTime = string.atof(string.split(file.readline())[0])
            for j in xrange(1, getSkip(expName) + 1/1):
                file.readline()
            endTime = string.atof(file.readline())
            end = file.readline() #skip the time line
            file.readline() #skip the blank row
#            print startTime, endTime
            print >>result, startTime, endTime
            pid = ""
            while True:
                l=cpumem.readline() #skip the time
#                print l
                l = cpumem.readline()
#                    print "atof", l
                l = l.strip()
                if len(l) == 0:
                    return
#                
#                print l
                recordTime =  string.atof(l)
#                print recordTime, startTime, endTime
                nextTime = nextStartTime(endTime, timeArray)
#                print "nextTime of %f"%endTime + " is %f"%nextTime
                line = cpumem.readline()
                if(recordTime >= startTime and recordTime <= endTime):
                    if(line.find("----") < 0):
                        if(line.find("openttd") >=0 and line.find("grep openttd") <0 and (pid == "" or line.find(pid) >=0 )):
                            words = string.split(line)
                            if pid == "":
                                pid = words[1]
#                            print recordTime, words[2], words[4]
                            print >> result, recordTime, words[2], words[4], words[9]
                        elif(line.find("win32")>=0):
                            words = string.split(line)
                            if len(words)<3:
                                break
                            print >> result, recordTime, words[1], words[2];
##                            print recordTime, words[0], words[1], words[2],words[3]
#                            print >> result, recordTime, words[0], words[1], string.atof(words[2]) + string.atof(words[3])
                    while(line.find("----") < 0):
                        line = cpumem.readline() 
                        if(len(line) == 0):
                            break; 
                    cpumem.readline()   #skip blank row
                elif recordTime < startTime:
                    while(line.find("----") < 0):
                        line = cpumem.readline()
                        if(len(line) == 0):
                            break; 
                    cpumem.readline()
                elif nextTime > 0 and recordTime > endTime and recordTime < nextTime:
                    while(line.find("----") < 0):
                        line = cpumem.readline()
                        if(len(line) == 0):
                            break; 
                    cpumem.readline()   #skip blank row
                else:
                    while(line.find("----") < 0):
                        line = cpumem.readline()
                        if(len(line) == 0):
                            break; 
                    cpumem.readline()
#                    print "end" , end  
                    break
            result.close()
        file.close()
#            print "*********************" + filename
    cpumem.close()
#    os.system("rm " + cpumempath)
#    print "rm " + cpumempath
    
def AnalysisNetwork(expId, expName, lowTurn, upperTurn, lowFactor, upperFactor):
    timeArray = preCompileTime(expName)
    dir = Configuration.getPathExp(expName)
    childDir = os.path.join(dir, "network")
    childDir = fm(childDir);
    File.newDir(childDir)
    packageResult = Configuration.getPackageResult(expId, expName, lowFactor)
    if os.path.exists(packageResult) == False:
        return
    network = open(packageResult)
    
    for pth in xrange(lowTurn, upperTurn + 1/1):
        filename = Configuration.getExpLog(expId, expName, lowFactor)
        print "***network*** " + filename
        file = open(filename)
        line = file.readline()
        if len(line) == 0:
            return 
        for i in xrange(1, string.atoi(line)  + 1):
            filename = Configuration.getNetworkResultFile(expId, expName, lowFactor)
            print filename
            result = open(filename, "w")
            print "start", file.readline()
            startTime = math.floor(string.atof(file.readline()))
            for j in xrange(1, getSkip(expName) + 1/1):
                file.readline()
            endTime = string.atof(file.readline())
            end = file.readline() #skip the time line
            file.readline() #skip the blank row
            print "#", startTime, endTime
#            print >>result, "#", startTime, endTime
            while True:
                nextTime = nextStartTime(endTime, timeArray)
                line = network.readline()
                if line == None : break
#                print line
                words = line.split()
                if len(words) == 0: break
                recordTime = string.atof(words[0])
                if(recordTime >= startTime and recordTime <= endTime):
#                    print recordTime, words[1], words[2]
                    # words[1]: packets count, words[2]: bytes 
                    print >> result, recordTime - startTime, words[1], words[2]
                elif recordTime < startTime:
                    continue
                elif nextTime > 0 and recordTime > endTime and recordTime < nextTime:
                    continue
                else:
                    print "recordTime > nextTime", recordTime, nextTime
                    break
            result.close()
        file.close()
        print "*********************" + filename
    network.close()

def getFloatCputime(cputime):
    retval = 0
    words = string.split(cputime, ":")
    l = len(words)
    for i in xrange(0, l/1):
        curval = string.atof(words[i])
        for j in xrange(0, l-i-1):            
            curval = curval * 60 
        retval = retval + curval
    return retval *1.0

def network(expId, expName, lowTurn, upperTurn, lowFactor, upperFactor, year, isServer = True):
    filename = Configuration.getPackageName(expId, expName, lowFactor)
    generateNetworkReport(expId, expName, lowFactor, filename)
    AnalysisNetwork(expId, expName, lowTurn, upperTurn, lowFactor, upperFactor)
    import Analysis2
    Analysis2.plotComparing(expId, expName, lowTurn, upperTurn, lowFactor, upperFactor, "", 0, 850 * year)
#    networkPrediction(expId, expName, lowTurn, upperTurn, lowFactor, upperFactor, "network")
"""
It use the tshark to generate the csv format. It use Capinfos to get the start time of 
measurement and then use the tShark to stastics the network result
"""
def generateNetworkReport(expId, expName, factor, packetFile):
    dir = Configuration.getPathExp(expName)
    hostname = getHostname()
    tmpPackages = fm(os.path.join(dir, "%d_tmpPackages"%expId + "_" + expName + "_" + hostname))
    packageResult = Configuration.getPackageResult(expId, expName, factor)
    if sys.platform == "win32":
        print "capinfos -S " + packetFile
        file = os.popen("capinfos -S " + packetFile)
    else:
        file = os.popen("capinfos -S " + packetFile)
    lines = file.readlines()
#    print lines
    if len(lines) == 0: return
    for line in lines:
        print line
    startTime = string.atoi(string.split(lines[8])[2])
    
    """
    Use tshark to generate the package and byte per second sequence
    """
    
    if sys.platform == "win32":
        os.system("c:\\\"program files\"\\wireshark\\tshark -q -z io,stat,1 -r "+ packetFile + " > " + tmpPackages)
    else :
        os.system("tshark -q -z io,stat,1 -r "+ packetFile + " > " + tmpPackages)
    
    resultFile = open(packageResult, "w")
    file = open(tmpPackages)
    line = file.readline()
    
    while line.find("Time") < 0 :
        line = file.readline()
    
    line = file.readline()
    
    time = startTime
    while line.find("==") < 0:
        words = line.split()
        print >>resultFile, time, words[1], words[2]
        time = time + 1
        line = file.readline()
    return True

def drawCPUGraph(expId, expName, lowTurn, upperTurn , low, upper, maxYear):
    try:
        import Analysis2
        Analysis2.plotComparing(expId, expName, lowTurn, upperTurn, low, upper, "", 0,  850 * maxYear)
        Analysis2.boxplot(expName, 1, lowTurn, upperTurn, low, upper, expId, False, "")
        Analysis2.boxplot(expName, 2, lowTurn, upperTurn, low, upper, expId, False, "")
        import AnalysisTime
        AnalysisTime.predictTimeLinearRegression(expId, expName, low, upper, False)
    except Exception:
        print traceback.print_exc()   

def getSimilityVal(expId, expName, lowTurn, upperTurn, lowFactor, upperFactor, filetype):
    v = []
    import AnalysisTime
    for factor in xrange(lowFactor, upperFactor+1/1):
        v = []
        for turn in xrange(lowTurn, upperTurn +1/1):
            v.append(readData(expId, expName, turn, factor, 1, filetype))
#            print len(v)
        for i in xrange(lowTurn, upperTurn+1/1):
            for j in xrange(i+1, upperTurn+1/1):
                print i,j, AnalysisTime.correlationOfTwoVector(v[i-1], v[j-1])

def getSimilityValTwoExp(expIda, expIdb, expName, turn, factor, col,filetype):
    v = []
    v.append(readData(expIda, expName, turn,factor, col, filetype))
    v.append(readData(expIdb, expName, turn,factor, col, filetype))
    import AnalysisTime
    print AnalysisTime.correlationOfTwoVector(v[0], v[1])
    
def networkPrediction(expId, expName, lowTurn, upperTurn, lowFactor, upperFactor, filetype):
    import AnalysisTime
    filenames  = ["packets", "bytes"]
    for col in [1,2]:
        y = []
        for factor in xrange(lowFactor, upperFactor+1/1):
            total = 0
            count = 0
            for turn in xrange(lowTurn, upperTurn+1/1):
                v = readData(expId, expName, turn, factor, col, filetype)
                mv = AnalysisTime.mean(v)
                total = total + mv
                count = count + 1
            total = total/count
            y.append(total)
        filename = "%d_"%expId + expName + "_" + filenames[col-1]+ "_prediction.png"
        path = fm(os.path.join(Configuration.path, expName, "network", filename))
        AnalysisTime.linearRegression(xrange(lowFactor, upperFactor+1/1),  y , path, 5, False)

def readData(expId, expName, turn, factor, col, filetype):
    filename = fm(os.path.join(Configuration.path, "trunk", "%d_"%expId + expName + "_%d_%d_"%(turn, factor) + filetype + ".txt"))
    file = open(filename)
    ret = []
    for line in file.readlines():
        words = string.split(line)
        ret.append(string.atof(words[col]))
#    print "read data " + filename, " ", len(ret)
    return ret
"""
split the csv result it into each experiment
"""


if __name__=="__main__":
    updateSkipList(1)
    print 1
#    File.newDir = str
#    def getHostname():
#        return "node306"
#    Configuration.getHostName = getHostname
#    def t(abc):
#        return "../das3_test_result/"
#    Configuration.getPath = 
#    Configuration.path = "D:\\RTSGameTrace\\das3_test_result"
    staticsCPUandMem(10000, "1a1", 1, 1, 1, 1, 1);
#    expId = Configuration.getId()
#    updateSkipList(1)
#    staticsCPUandMem(expId, "1c", 1,1,1,4,1)
#    drawCPUGraph(expId, "1c", 1, 1, 1, 4, 1)
#    network(Configuration.getId(), "1c", 1,1, 1, 4, 1)
#    getSimilityVal(expId, "1c", 1, 3, 1, 4, "network")
#    updateSkipList(1)
#    getSimilityValTwoExp(1, 32, "1a1", 1,1, 1,"cpu")
    
    
    
#    import Analysis2
#    Analysis2.multiPlotData(expId, "1c", 1, 3, 1, 4, "network")
#    networkPrediction(expId, "1c", 1, 3, 1, 4, "network")
#    getSimilityVal(32, "1a1", 1, 1, 1, 16, "cpu")
#    staticsCPUandMem(expId, "1c", 1,3,1,3,3)
#    drawCPUGraph(expId, "1c", 1, 3, 1, 4, 3)
#    network(Configuration.getId(), "1c", 1, 3, 1, 4,3)
#    AnalysisNetwork(expId, "1c", 1, 3, 1, 4)

